var citiesList = "sh";

var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});
//var cities = citiesList.split(", ");

//var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201/Communities';
var async = require('async');

async.eachSeries(cities, function (city) {
    //var url = 'http://' + city + '.anjuke.com/community/view/';
    //var url = 'http://sh.58.com/xiaoqu/yangguanghuacheng/'
    var dbCollection = 'xq_58_' + city;
    var dbCollectionDetail = 'xq_58_' + city + '_detail';

    function run() {
        var manifest = {
            target: {
                url: '',
                ranges: {
                    // number: [1, 1976, 1], // from 0 to 100 and step 5 numbers each time
                    // letter: ['a', 'z'] // from a to z, put {{letter}} where you want it
                    // resource: 'csv' // configure this resource and put {{resource:csv}}
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            resources: {
                mongodb: {
                    plugin: 'mongodb',
                    options: {
//                collection: 'xq_shanghai',
                        collection: dbCollection,
                        field: 'commId'
                    }
                }
            },
            plugins: {
                mongodb: {
                    connection: connection
                },
                csv: true
            },

            extractors: [{
                name: 'details',
                root: '//d1[@class="bhrInfo"]',
                //root:'',
                fields: {
                    developedBy: '//dd[6]/text()',
                    propertyType: '//dd[5]/text()[2]',
                    managedBy: '//dd[8]/text()',
                    fees: '//dd[7]/span[@class="ddinfo"]/text()',
                    rentalPercentage:'//*[@id="peitao_3"]/ul/li[1]/span[3]',
                    greenPercentage:'//*[@id="peitao_3"]/ul/li[2]/span[3]',
                    constructionArea: '//*[@id="peitao_3"]/ul/li[4]/span[3]',
                    area:'//*[@id="peitao_3"]/ul/li[3]/span[3]',
                    dateBuilt: '//dd[5]/text()[1]',
                    parkingUnits: '//*[@id="peitao_3"]/ul/li[5]/span[3]',
                }
            }]
        };

        var MongoClient = require('mongodb').MongoClient, assert = require('assert');
        var dbconn = null;

        function insertDocuments(doc, callback) {

            var collection = dbconn.collection(dbCollectionDetail)

            collection.insert(
                [{
                    commId: doc.commId,
                    name: doc.name,
                    district: doc.district.trim(),
                    subdistrict: doc.subdistrict.trim(),
                    address: doc.address,
                    developedBy: doc.developedBy,
                    managedBy: doc.managedBy,
                    propertyType: doc.propertyType,
                    fees: doc.fees,
                    constructionArea: doc.constructionArea,
                    units: doc.units,
                    dateBuilt: doc.dateBuilt,
                    rjl: doc.rjl,
                    parkingUnits: doc.parkingUnits,
                    greenPercentage: doc.greenPercentage,
                    rentalPercentage: doc.rentalPercentage,
                    desc: doc.desc,
                    url: doc.url
                }]
                , function (err, result) {
                    //assert.equal(err, null);
                    //assert.equal(1, result.result.n);
                    //assert.equal(1, result.ops.length);
                    console.log("Inserted %s into the document collection", doc.name);
                    callback(result);
                });
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
            console.log(data.data);

            data.data.forEach(
                function (doc) {
//                   insertDocuments(doc, function(){});
                })
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            console.log('The job is done');
            // not exactly,
        });

        function findCommids(db, callback) {
            var collection = db.collection(dbCollection);
            collection.find({}).toArray(
                function (err, docs) {
//            console.log(docs);
                    callback(docs);
                });
        };

        var ids = [];

        function init(callback) {
            MongoClient.connect(connection,
                function (err, db) {
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    dbconn = db;
                    //collection = db.collection(dbCollectionDetail)
                    callback(null, 'init');
                });
        }

        function getIds(callback) {
            findCommids(dbconn, function (docs) {
                    ids = docs;
                    callback(null, 'getIds');
                }
            );
        }

        function start(callback) {
            console.log("starting...");
            var cnt = 0;

            async.eachSeries(ids, function (item, callbackInner) {
                    manifest.target.url = item.href;

                    //if (item.href=='http://sh.58.com/xiaoqu/shimaobinjianghuayuan/'){
                    //    callbackInner()
                    //    return
                    //}
                    console.log(manifest.target.url);


                    var job1 = new Crawler(manifest);
                    console.log(job1)

                    try{
                        job1.on('data', function (data) {
                            console.log(data)
                            data.data.forEach(function (doc) {
                                console.log(doc.greenPercentage);

                                doc.url = manifest.target.url;
                                //insertDocuments(doc, function () {
                                //});
                            })
                        });

                        job1.on('error', function (err) {
                            throw err;
                        });

                        job1.on('end', function () {

                            console.log('The job is done: %d', ++cnt);
                            delete job1;

                            callbackInner();
                        });

                        job1.start();

                    }catch(e){
                        console.log(e)
                        return
                    }

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }
                }
            );
        }

        function finish(callback) {
            console.log("closing db...");
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);

            });
    }

    run();
});
